{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "withwandb.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "c86c7fab16c4455a80cd4ed2248616b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_74bcf26429224b719d496dad48e09f2b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c999b48c418f406383bd974cb6c354ed",
              "IPY_MODEL_9af13c2275184f40ba50cae9a11238f2"
            ]
          }
        },
        "74bcf26429224b719d496dad48e09f2b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c999b48c418f406383bd974cb6c354ed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f1efb7c2004c459db6e2879d77310696",
            "_dom_classes": [],
            "description": " 15%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 20,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 3,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b8ce7b4f8e4c4634a023da584285e470"
          }
        },
        "9af13c2275184f40ba50cae9a11238f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f692613d341f4bd295196751e2a62b68",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 3/20 [07:04&lt;40:10, 141.82s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7b58eb1cf4634ea6a6c6269de5748889"
          }
        },
        "f1efb7c2004c459db6e2879d77310696": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b8ce7b4f8e4c4634a023da584285e470": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f692613d341f4bd295196751e2a62b68": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7b58eb1cf4634ea6a6c6269de5748889": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HcPcj3eDspSM",
        "outputId": "89d2ae38-3723-4bee-e2b3-a7e3f4d2156a"
      },
      "source": [
        "pip install transformers"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/fd/1a/41c644c963249fd7f3836d926afa1e3f1cc234a1c40d80c5f03ad8f6f1b2/transformers-4.8.2-py3-none-any.whl (2.5MB)\n",
            "\u001b[K     |████████████████████████████████| 2.5MB 4.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/75/ee/67241dc87f266093c533a2d4d3d69438e57d7a90abb216fa076e7d475d4a/sacremoses-0.0.45-py3-none-any.whl (895kB)\n",
            "\u001b[K     |████████████████████████████████| 901kB 26.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (4.6.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/e2/df3543e8ffdab68f5acc73f613de9c2b155ac47f162e725dcac87c521c11/tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3MB 28.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from transformers) (3.13)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Collecting huggingface-hub==0.0.12\n",
            "  Downloading https://files.pythonhosted.org/packages/2f/ee/97e253668fda9b17e968b3f97b2f8e53aa0127e8807d24a547687423fe0b/huggingface_hub-0.0.12-py3-none-any.whl\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.5.30)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Installing collected packages: sacremoses, tokenizers, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.0.12 sacremoses-0.0.45 tokenizers-0.10.3 transformers-4.8.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0nGwsgVksqU0"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dcMCFelTsvBu"
      },
      "source": [
        "import json\n",
        "from transformers import GPT2LMHeadModel, GPT2Tokenizer\n",
        "from transformers.data.data_collator import DataCollatorForLanguageModeling\n",
        "import torch\n",
        "from torch.utils.data import Dataset\n",
        "from torch.utils.data import DataLoader\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fotj_cXOVNiV"
      },
      "source": [
        "\n",
        "import random\n",
        "import numpy as np\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "from tqdm.notebook import tqdm\n",
        "import numpy as np\n",
        "device = 'cuda'"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gbe0liIcVf9u"
      },
      "source": [
        "%%capture\n",
        "!pip install wandb --upgrade"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0wCSfpJAVnJ1",
        "outputId": "9e4befd7-02d4-4960-f307-843bd8e23b7d"
      },
      "source": [
        "import wandb\n",
        "\n",
        "wandb.login()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mromanovand\u001b[0m (use `wandb login --relogin` to force relogin)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "njh7bCxfW3-Y"
      },
      "source": [
        "class TerraDataset(Dataset):\n",
        "    def __init__(self, file):\n",
        "        self.file = file\n",
        "        self.strings = self.load()[0]\n",
        "        self.predict = torch.tensor(self.load()[1])\n",
        "        self.tokens = self.transform()\n",
        "    def load(self):\n",
        "      data = [json.loads(d) for d in open(self.file, encoding=\"utf-8\")]\n",
        "      s1 = []\n",
        "      s2 = []\n",
        "      for a in data:\n",
        "        s1.append( a['premise']+' ~> '+a['hypothesis'])\n",
        "        if a['label']== 'entailment':\n",
        "          s2.append(1)\n",
        "        else:\n",
        "          s2.append(0)\n",
        "      return [s1,s2]\n",
        "    def transform(self):\n",
        "      model_name_or_path = \"sberbank-ai/rugpt3small_based_on_gpt2\"\n",
        "      tokenizer = GPT2Tokenizer.from_pretrained(model_name_or_path)\n",
        "      input_ids = []\n",
        "      for i,string in enumerate(self.strings):\n",
        "        input_ids.append(tokenizer(string, return_length=True))\n",
        "      return input_ids\n",
        "    def __len__(self):\n",
        "        return len(self.strings)\n",
        "    def __getitem__(self, idx):\n",
        "        return self.tokens[idx],self.predict[idx]"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VfkDyqoKs8Zk"
      },
      "source": [
        "cuda = 'cuda'"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Id_zUCSkWvu-"
      },
      "source": [
        "class LogisticRegression(torch.nn.Module):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.gpt3 = GPT2LMHeadModel.from_pretrained(\"sberbank-ai/rugpt3small_based_on_gpt2\").to(device)\n",
        "        self.linear = torch.nn.Linear(768,1).to(device)\n",
        "    def forward(self, sampler):\n",
        "        a = self.gpt3(input_ids = torch.tensor(sampler['input_ids']).to(device), attention_mask = torch.tensor(sampler['attention_mask']).to(device),\n",
        "                return_dict = True, output_hidden_states = True)['hidden_states'][-1]\n",
        "        l = sampler['length'].to(cuda)-1\n",
        "        l = l.unsqueeze(-1)\n",
        "        ind = l.repeat(1,768)\n",
        "        ind = ind.unsqueeze(1)\n",
        "        res = torch.gather(a, 1, ind)\n",
        "        predicted = torch.sigmoid(self.linear(res))\n",
        "        return predicted"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2aQIzq06VyXA"
      },
      "source": [
        "config = dict(\n",
        "    epochs=20,\n",
        "    batch_size=10,\n",
        "    learning_rate=0.00001,\n",
        "    )"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FOOgdN9IWhQ3"
      },
      "source": [
        "def model_pipeline(hyperparameters):\n",
        "    with wandb.init(project=\"pytorch-demo\", config=hyperparameters):\n",
        "      config = wandb.config\n",
        "      model, train_loader, criterion, optimizer,val_loader,length = make(config)\n",
        "      train(model, train_loader,  criterion, optimizer, config,val_loader,length)\n",
        "    return model"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CHi1daGgWhZU"
      },
      "source": [
        "def make(config):\n",
        "    train_loader = make_loader( batch_size=config.batch_size)\n",
        "    val_loader,length = valid_loader( batch_size=config.batch_size)\n",
        "    model = LogisticRegression().to(device)\n",
        "    criterion = torch.nn.BCELoss()\n",
        "    optimizer = torch.optim.Adam(\n",
        "        model.parameters(), lr=config.learning_rate)\n",
        "    return model, train_loader, criterion, optimizer, val_loader,length"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1HN8boNOWhen"
      },
      "source": [
        "def make_loader( batch_size):\n",
        "    training_data = TerraDataset('/content/drive/MyDrive/Colab Notebooks/TERRa/train.jsonl')\n",
        "    tokenizer = GPT2Tokenizer.from_pretrained(\"sberbank-ai/rugpt3small_based_on_gpt2\")\n",
        "    tokenizer.add_special_tokens({'pad_token': '[PAD]'})\n",
        "    collator = DataCollatorForLanguageModeling(tokenizer = tokenizer, mlm=False)\n",
        "    def coll(s):\n",
        "      a,b = map(list,zip(*s))\n",
        "      return collator(a),b\n",
        "    loader = torch.utils.data.DataLoader(training_data,\n",
        "                                         batch_size=batch_size, \n",
        "                                         shuffle=True, collate_fn = coll\n",
        "                                         )\n",
        "    return loader\n",
        "\n",
        "def valid_loader( batch_size):\n",
        "    valid_data = TerraDataset('/content/drive/MyDrive/Colab Notebooks/TERRa/val.jsonl')\n",
        "    tokenizer = GPT2Tokenizer.from_pretrained(\"sberbank-ai/rugpt3small_based_on_gpt2\")\n",
        "    tokenizer.add_special_tokens({'pad_token': '[PAD]'})\n",
        "    collator = DataCollatorForLanguageModeling(tokenizer = tokenizer, mlm=False)\n",
        "    def colla(s):\n",
        "      a,b = map(list,zip(*s))\n",
        "      return collator(a),b\n",
        "    loader = torch.utils.data.DataLoader(valid_data,\n",
        "                                         batch_size=batch_size, \n",
        "                                         shuffle=True, collate_fn = colla\n",
        "                                         )\n",
        "    length = valid_data.__len__()\n",
        "    return loader, length"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uGOU6t7vXFqY"
      },
      "source": [
        "def train(model, loader, criterion, optimizer, config,val_loader,length):\n",
        "\n",
        "    batch_ct = 0\n",
        "    best = 0 \n",
        "    for epoch in tqdm(range(config.epochs)):\n",
        "        for batch_ndx, (sample, target) in enumerate(loader):\n",
        "            loss = train_batch(sample, target, model, optimizer, criterion)  \n",
        "            batch_ct += 1\n",
        "            if ((batch_ct + 1) % 5) == 0:\n",
        "                train_log(loss, epoch,batch_ct)\n",
        "        accur, val_loss = val_test(model, val_loader,length,criterion)\n",
        "        if accur > best:\n",
        "          best = accur\n",
        "          best_model_state = model.state_dict()\n",
        "        val_log(accur,val_loss,batch_ct)\n",
        "        torch.save(best_model_state, '/content/drive/MyDrive/Colab Notebooks/finetunemodel.pth')\n",
        "\n",
        "def train_batch(sample, target, model, optimizer, criterion):\n",
        "    y_predicted = model(sample).squeeze()\n",
        "    y_predicted = y_predicted.float().to(device = cuda)\n",
        "    target = torch.tensor(target)\n",
        "    target = target.float().to(device = cuda)\n",
        "    loss = criterion(y_predicted, target)\n",
        "    optimizer.zero_grad()\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "    return loss"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lLOF1n6Qkel8"
      },
      "source": [
        "def val_test( modus,valid_dataloader,length,loss_func):\n",
        "  k = 0\n",
        "  cor_pred = []\n",
        "  val_loss = []\n",
        "  for ind , (valid,target) in enumerate(valid_dataloader):\n",
        "    target = torch.tensor(target)\n",
        "    target = target.float().to(device = cuda)\n",
        "    y = modus(valid).squeeze().to(cuda)\n",
        "    y = y.float().to(device = cuda)\n",
        "    answ = torch.where(y > 0.5, 1. , 0.)\n",
        "    cor_pred.append(torch.sum(torch.eq(target, answ).gt(0).to(torch.float32)).item())\n",
        "    val_loss.append(float(loss_func(y,target)))\n",
        "  result = sum(cor_pred)/length\n",
        "  return result,np.mean(val_loss)"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZDV8kZcAXKPh"
      },
      "source": [
        "def train_log(loss, epoch, batch_ct):\n",
        "    loss = float(loss)\n",
        "    wandb.log({\"epoch\": epoch, \"loss\": loss},step = batch_ct)\n",
        "def val_log(accur,val_loss,batch_ct):\n",
        "    wandb.log({\"val_accur\": accur, \"val_loss\": val_loss},step = batch_ct)"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 208,
          "referenced_widgets": [
            "c86c7fab16c4455a80cd4ed2248616b7",
            "74bcf26429224b719d496dad48e09f2b",
            "c999b48c418f406383bd974cb6c354ed",
            "9af13c2275184f40ba50cae9a11238f2",
            "f1efb7c2004c459db6e2879d77310696",
            "b8ce7b4f8e4c4634a023da584285e470",
            "f692613d341f4bd295196751e2a62b68",
            "7b58eb1cf4634ea6a6c6269de5748889"
          ]
        },
        "id": "YllTYnzMtXBw",
        "outputId": "3590e26b-3e6b-41a8-8c05-63252fc509ee"
      },
      "source": [
        "mod = model_pipeline(config)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                Tracking run with wandb version 0.10.33<br/>\n",
              "                Syncing run <strong style=\"color:#cdcd00\">northern-surf-38</strong> to <a href=\"https://wandb.ai\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
              "                Project page: <a href=\"https://wandb.ai/romanovand/pytorch-demo\" target=\"_blank\">https://wandb.ai/romanovand/pytorch-demo</a><br/>\n",
              "                Run page: <a href=\"https://wandb.ai/romanovand/pytorch-demo/runs/1g9f318r\" target=\"_blank\">https://wandb.ai/romanovand/pytorch-demo/runs/1g9f318r</a><br/>\n",
              "                Run data is saved locally in <code>/content/wandb/run-20210714_133530-1g9f318r</code><br/><br/>\n",
              "            "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c86c7fab16c4455a80cd4ed2248616b7",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  import sys\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "keSorMavtgnR"
      },
      "source": [
        ""
      ],
      "execution_count": 17,
      "outputs": []
    }
  ]
}